{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST example for learning rate finder with mixed precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/davidtvs-nb/lr-finder-rev/py3/pytorch-lr-finder/env/lib/python3.6/site-packages/torch_lr_finder/lr_finder.py:5: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Subset, DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "try:\n",
    "    from torch_lr_finder import LRFinder\n",
    "except ImportError:\n",
    "    # Run from source\n",
    "    import sys\n",
    "    sys.path.insert(0, '..')\n",
    "    from torch_lr_finder import LRFinder\n",
    "\n",
    "from apex import amp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "SEED = 0\n",
    "\n",
    "def reset_seed(seed):\n",
    "    \"\"\"\n",
    "    ref: https://forums.fast.ai/t/accumulating-gradients/33219/28\n",
    "    \"\"\"\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ConvNet, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.conv2_drop = nn.Dropout2d()\n",
    "        self.fc1 = nn.Linear(320, 50)\n",
    "        self.fc2 = nn.Linear(50, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(F.max_pool2d(self.conv1(x), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
    "        x = x.view(-1, 320)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.dropout(x, training=self.training)\n",
    "        x = self.fc2(x)\n",
    "        return F.log_softmax(x, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = '../data'\n",
    "\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.1307,), (0.3081,))\n",
    "])\n",
    "\n",
    "trainset = datasets.MNIST(data_folder, train=True, download=True, transform=transform)\n",
    "# testset = datasets.MNIST(data_folder, train=False, download=True, transform=transform)\n",
    "\n",
    "DESIRED_BATCH_SIZE = 256"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Warm up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d275007799c43e2aca3b29f8b29f960",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    }
   ],
   "source": [
    "reset_seed(SEED)\n",
    "trainloader = DataLoader(trainset, batch_size=DESIRED_BATCH_SIZE, shuffle=True)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "model = ConvNet()\n",
    "model = model.to(device)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.5)\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device='cuda')\n",
    "lr_finder.range_test(trainloader, end_lr=10, num_iter=10, step_mode='exp')\n",
    "lr_finder.reset()\n",
    "\n",
    "del optimizer, model, lr_finder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LRFinder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Without `apex.amp`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32dfb9bbe0814bd8ad145d2deff97a51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXxU9b3/8ddnMpM9RAhhMeygCMhqVBQrdBO1rRS1/Fzq3lottnptrdfe9rbXXtvb2tLWalXc0Bbbe1Wq1q5qsRRRZBUURFlcWDRhT1gy2/f3xwwpjcmQQM6cnMn7+XjM4zFz5nvOfDjfYT75fr/nfL/mnENERKQlIb8DEBGRjk2JQkREMlKiEBGRjJQoREQkIyUKERHJSIlCREQyCvsdQFt1797dDRgwwO8wREQCZcmSJVudc5WHs2/gEsWAAQNYvHix32GIiASKmb1zuPuq60lERDJSohARkYyUKEREJCMlChERyUiJQkREMlKiEBGRjJQoRMQXqzbvpiGe8DsMaQUlChHJuh17onzmzvnc9NgKv0ORVlCiEJGse/ODOhJJx9Ovbuap5Zv8DkcOQYlCRLJubW09AEN6lPKt373Gxh17fY5IMlGiEJGsW1tTT1Ekjwcuq8YBN/7fqySSWpa5o1KiEJGsW1tTz+AeJfSvKOG754zglQ3bmTlvvd9hSQuUKEQk69bV1DOkshSA88ZVcfbIXvzkr2tY9u4OnyOT5ihRiEhW7WmIs3nXfob0SCUKM+MHU0fRs0sh1z26jJ17oz5HKE0pUYhIVq07aCD7gPLiCHddPI6auv18/bEVOKfxio5EiUJEsmptzYcTBcCYvkdxy1nDeG71Bzwwf4MfoUkLlChEJKvW1tQTDhn9K0o+9N4VEwYweURP/udPb7BU4xUdhhKFiGTV2pp6+lcUE8n78M+PmfGj80fT+6hCps9eyrb6Bh8ilKaUKEQkq9bV1n+o2+lg5UUR7r74BLbtiXL9b5fr/ooOwLNEYWZ9zWyuma0ys9fN7PpmypSb2e/N7NV0mSu8ikdE/BdLJHln214GV7acKACOryrnv6ccz/y1W5nx7JosRSct8bJFEQe+5pwbDowHppvZ8CZlpgOrnHOjgUnAT8ws38OYRMRH72zbQzzpMrYoDph2Yl8uOLEvd81dx7OrPshCdNISzxKFc26Lc25p+nkdsBqoaloMKDMzA0qB7aQSjIjkoJaueGrJd88Zwciqcm78v+Vs2LrHy9Akg6yMUZjZAGAssLDJW3cCw4DNwErgeudcspn9rzazxWa2uLa21uNoRcQrBxLFobqeDiiM5PHLi8cRDhlXPbyIXXtjXoYnLfA8UZhZKfAEcINzbneTtycDy4GjgTHAnWbWpekxnHMznXPVzrnqyspKr0MWEY+srann6PJCSgrCrd6nb7di7r2kmve272X6o0uJJT70t6R4zNNEYWYRUklitnNuTjNFrgDmuJS1wAbgOC9jEhH/rK2tZ3Aru50OdtLAbnx/6kjmr93Krb9f5UFkkomXVz0Z8ACw2jk3o4Vi7wIfT5fvCQwFNIWkSA5KJh3rava0enyiqc9V9+VLpw/iVy+/w8ML3m7f4CSj1rf/2m4CcAmw0syWp7d9E+gH4Jy7B/geMMvMVgIG3Oyc2+phTCLik8279rEvljjsRAHwjTOPY13tHm59ZhUDu5dw+rHqis4GzxKFc24+qR//TGU2A2d4FYOIdByNVzy1ciC7OXkh4+cXjOG8uxcwffZSfjf9VIb0KGuvEKUFujNbRLKirZfGtqSkIMwDl59IQSSPK2ctZvseTUvuNSUKEcmKdbX1dC2OUFFacMTHqjqqiJmXnsD7u/dzza+XEI3rSigvKVGISFbU1jXQq7yo3Y43rl9Xbj9/FK9s2M4tc1ZqDQsPeTmYLSLSKJpw5Ifb92/TKWOq2LB1Dz977i36VxTz1Y8f067HlxQlChHJilg8SX5exutbDsv1Hz+Gd7fvZcazb9K3WxFTx/Zp98/o7JQoRCQr4slks2tQHCkz43/OHcWWnfv5xuMr6F1exPhBFe3+OZ2ZxihEJCuiCUfYg0QBkB8Occ/nT6Bft2K+9KslvPVBnSef01kpUYhIVnjV9XRAeXGEWVecRH44xKUPvsKmnfs8+6zORolCRLIilvCm6+lgfbsV88iVJ1HfEOfSBxbqHot2okQhIlkRTzrPEwXAsN5duP/Sat7bsY8rZi1iT4OWuDlSShQikhXReJKwh11PBzt5UAV3XjiWlRt38qVfLWF/LJGVz81VShQikhWxRJL8LLQoDjhjRC9+eN4o5q/dyrW/XkJDXMnicClRiEhWZGOMoqnPVffl+1NHMndNLdNnL9NUH4dJiUJEsiKeyM4YRVMXndyPW6eM4LnVH3D9b5dphbzDoEQhIlkRTSSJZGmMoqlLTxnAtz41jD+99j43/Ha5kkUb6c5sEckKP7qeDvaFjwzCObjtj6tJJB13XDi23eeeylU6SyLiuUTSkXT4migAvnj6IL796eH8+fX3mf7oUo1ZtJIShYh47kBXTyTsT9fTwa46bSDf/cxwnl31AV+erUtnW0OJQkQ815goQh3jJ+fyCQP53meP57nVNVw5axH1uikvo45RayKS02KJ1KJCfg1mN+eS8f2ZMW00Czds5+L7F7Jzr6b7aIkGs0XEc//seupYf5ueO64PpQVhrvvNMqbd+xJfOG0QKzbtZNm7O3lv+15uOvM4Lhnf3+8wfdexak1EclJjovB5MLs5Z4zoxawrTmTTjn1844kVPLVsM12L8zmuVxe+/eRr/Oy5Nzv9MqtqUYiI5zpi19PBTh3cnbk3TWLX3hiDK0sJhYx4Ism/z1nJz557i231Ub57zgjyQh0zfq8pUYiI5zpyi+KAHmWF9CgrbHwdzgtx+/mjqCjJ595569m+N8qMaaMpCOf5GKU/lChExHMH7lfoyImiOWbGLWcPo6I0n+//8Q1q6xq475JqyosjfoeWVcGqNREJpHgy1fWUzdlj29PVpw/m5xeMYfm7OznvngVs3LHX75CyKpi1JiKBcqDrKVvrUXhhypgqHrnqJGp272fqLxewcuMuv0PKGiUKEfFcLKBdT02NH1TBE9eeSn5eiM/du4A/rtzid0hZ4VmtmVlfM5trZqvM7HUzu76FcpPMbHm6zN+9ikdE/BMNwGB2ax3Ts4wnp09gxNHlfHn2Un7+3Fs5f/msl7UWB77mnBsOjAemm9nwgwuY2VHAL4FznHMjgM95GI+I+CSeCPYYRVOVZQXM/sLJnDu2ip8+9yZf+c2ynF6b27Nac85tcc4tTT+vA1YDVU2KXQTMcc69my5X41U8IuKfXBijaKowksdPpo3m5jOP4w8rt3D2Hf9gyTs7/A7LE1lJ72Y2ABgLLGzy1rFAVzN7wcyWmNmlLex/tZktNrPFtbW13gYrIu0ul7qeDmZmXDtpMP979SnEE47P3bOAGX9dk3MLI3lea2ZWCjwB3OCc293k7TBwAvApYDLwbTM7tukxnHMznXPVzrnqyspKr0MWkXYWy7Gup6ZOGtiNP9/wEaaO7cMdf1vLeXcv4K0P6vwOq914WmtmFiGVJGY75+Y0U2Qj8Bfn3B7n3FZgHjDay5hEJPviHWg9Cq+UFUb4ybTR3H3xON7bvpdP/WI+981bTyIZ/IFuL696MuABYLVzbkYLxZ4CTjOzsJkVAyeTGssQkRzSOEbRQdaj8NJZI3vz13+byMRjK7ntj6u5YOZLvLNtj99hHREva20CcAnwsfTlr8vN7Gwzu8bMrgFwzq0G/gysAF4B7nfOveZhTCLig2iOdz01VVlWwMxLTmDGtNG88X4dk382jwfnbyAZ0NaFZ3M9OefmA4dsZzrnbgdu9yoOEfFfR1oKNVvMjHPH9eHUwd355u9Wcuszq/jTa1v40fmjGdi9xO/w2qRzpHcR8VU8R696ao1e5YU8cFk1P/ncaNa8X8eZAWxddL5aE5GsO9D1FO6k6zmYGeed0Idnb5zIhCHdufWZVVx438u8tz0YkwsqUYiI52KJJJE8I3WNS+fVs0uqdfGj80exavNuJv9sHr966e0O37pQohARz8XiyU7Z7dQcM2NadV/+/G+nc0L/rnz7qdc5/54FrHm/4953oZoTEc/Fk06Joomqo4p45MqTmDFtNG9v28un7vgHP/7LGvbHEn6H9iGqORHxXDTd9ST/6sCVUc/dOJEpY6q4c+5azvjpPOau6VjT3ilRiIjn1PWUWbeSfH4ybTSPfvFkInnGFQ8t4tpfL2Hzzn1+hwYoUYhIFqQGs/VzcyinDu7On64/nZsmD+Vvb9Qw6ccv8J2nXvM9YajmRMRzsaRT11Mr5YdDTP/oEJ67cSJTx1Qxe+G7TLx9LrfMWenb5bRKFCLiOXU9tV3fbsX88PxRvHDTJKZV9+WJJRt56MW3fYnFsyk8REQOUNfT4evTtZjbpo7kuo8N8W2uLCUKEfFcLKGupyPVu7zIt89WihcRz0XVogg01ZyIeC6uRBFoqjkR8Zy6noJNiUJEPKfB7GBTzYmI56KJJJGwfm6CSjUnIp6LJxyRTroWRS5QohARz6nrKdhUcyLiuZi6ngJNNScinovGk77dVSxHTjUnIp6LJ12nXS87FyhRiIjn1PUUbKo5EfGUcy59w51+boJKNScinoolHAD5ujM7sJQoRMRT8WQSgLBaFIGlmhMRT8XiqRaFup6CSzUnIp6KJlItCnU9BZdnicLM+prZXDNbZWavm9n1GcqeaGZxMzvfq3hExB+xdKJQiyK4vFzhLg58zTm31MzKgCVm9qxzbtXBhcwsD/gh8FcPYxERn8TTg9kaowguz2rOObfFObc0/bwOWA1UNVP0K8ATQI1XsYiIf6KNLQp1PQVVVlK8mQ0AxgILm2yvAqYCdx9i/6vNbLGZLa6trfUqTBHxQKxxjEItiqDyvObMrJRUi+EG59zuJm//DLjZOZfMdAzn3EznXLVzrrqystKrUEXEAxqjCL5WjVGY2WBgo3OuwcwmAaOAR5xzOw+xX4RUkpjtnJvTTJFq4LdmBtAdONvM4s65J9vwbxCRDizWOEahrqegam2KfwJImNkQYCbQF3g00w6W+vV/AFjtnJvRXBnn3EDn3ADn3ADgceDLShIiuUVdT8HX2queks65uJlNBX7hnPuFmS07xD4TgEuAlWa2PL3tm0A/AOfcPYcVsYgESmPXkyYFDKzWJoqYmV0IXAZ8Jr0tkmkH59x8oNVtTefc5a0tKyLBoTGK4GttzV0BnALc5pzbYGYDgV95F5aI5IrGMQqtRxFYrWpRpG+S+yqAmXUFypxzP/QyMBHJDY1jFOp6CqxW1ZyZvWBmXcysG7AUuM/Mmh2gFhE5mLqegq+1NVeevgfiXFKXxZ4MfMK7sEQkV/xz9lh1PQVVaxNF2Mx6A9OAZzyMR0RyTCypFkXQtbbmbgX+Aqxzzi0ys0HAW96FJSK5IhZXogi61g5mPwY8dtDr9cB5XgUlIrnjwFVP6noKrtYOZvcxs9+ZWU368YSZ9fE6OBEJvqgGswOvtTX3EPA0cHT68fv0NhGRjOIJLYUadK2tuUrn3EPOuXj6MQvQNK4ickixRJKQQZ5uuAus1iaKbWb2eTPLSz8+D2zzMjARyQ2xRFKtiYBr7VxPVwK/AH4KOGABcLlHMXliyTvbmTlvPeFQiFDIGqcTiCaSxOJJYokkSQdJ50g6RyLpSCYhnkySSDrMjPxwiIJwiHDIaIgn2RdLsC+aSP/FZITMMIP0tOktMkiXI7UPqX3yQkYkz8gP55GfF6Igkvq8wkjqdePxSS0rWRTJozg/j8L8PMIhO+i4qX9fOC/UuD1x4N/kHCEzInkh8tLvRRNJGmJJookkSecwjFA6vqSDRNLhXOocFOfnUZwfpjg/j7LCMOVFEcqLIpQVRnTnrTQrmkhq5tiAa+1VT+8A5xy8zcxuILXwUCDUNyR4Z9te4snUD2Yi6XA48vNCRNKPUCj1A5mX/tHPCxkFkTAhM5LOEY0nqW+IE084CsIhSgvCVJYWEMkL4UglloRzGeNIve1wLpVxnXMk088TySTReJJd+2JE40ka4gkaYkka0s+dS5V3pPp9DwwSdhR5IaMokkdhJERxfpjSgjClhWHKCsJ0LcmnoiSfbiX5dC3Op6QgTElBHqUFYcoKI3QpSiWdokjeIROtBEs84TRzbMC1tkXRnBsJUKKYeGwlE4/NrWGVeCLdqokl0n/1pxJOMp0IDyTFpHOEQ5ZOhNaYKOPJJM7R2FLKD6daLanjpBJYKmlCKJTab280wd5onL3RBHX7Y+zaF2PX3hh1++PsiyXYH0uyP55qadXtj1PfEGPLrv2s2rKbbXuiROOZk1t+XojupflUdimkR1kB3YrzKcpPtZyKInkclU44FSX5VJQW0L00ny6FEULq/+6wYomkJgQMuCNJFKp5n4XzQpTlhSgrzDjje4fhnGNPNMGOPVH2ROPsaYhT35Cgfn+c3emks3NvjNq6Bmrq9vPe9r28+t7OdAJKNF6P31Q4ZHQryadnl0L6diuib9di+nQrZlD3EgZ2L6FXl0IlEh9FNUYReEeSKDL3sYg0YWap7qiCw/vaReNJdu6Lsq0+yvY9UbbWN7CtPsq2PQ1srYuyZfd+3thSx3Orav6lW64okseA7iUMqChmQPcSBlaUcEzPUob17kJhJK+9/nnSgljCafwq4DL+jzWzOppPCAYUeRKRSAvywyF6lBXSo6wwY7lk0vFB3X42bN3D+trUY8PWeta8X8ezqz4gnkx9pfNCxpDKUkZUdeHEAd04aWA3BnUv0RhJO4snkrorO+AyJgrnXFm2AhFpL6GQ0bu8iN7lRZw6uPu/vBdPJNm0cx+rt9SxavMuXtu8m3lv1jJn6SYAKssKmDC4grNH9ub0YyvV4mgHqTEKtSiC7Ei6nkQCJ5wXon9FCf0rSjjz+F5Aauxk/dY9LFy/nYUbtvHCm7U8uXwzZQVhPjm8J1PGVnHakO66YewwRXXVU+ApUUinZ2YMrixlcGUpF53cj1giyYJ12/jDis38+bX3mbNsEz27FDB1bB/OP6EPQ3qU+h1yoMTiSfLV9RRoShQiTUTyQo2XU3/vs8fzt9U1PL5kI/f9Yz33zlvHuWP7cNPkofQqzzxWIinxpK56CjolCpEMCsJ5nDWyN2eN7E1N3X4emL+Bh+a/zR9Wbubq0wfzpdMHUXKYV3F1FtGEoyhfiSLIVHsirdSjrJBbzhrG81+byMeH9eSO59/ijJ/O4+9v1vodWoemrqfgU6IQaaO+3Yq566JxPHbNKRRGQlz24Ct84/FX2bUv5ndoHZImBQw+1Z7IYTpxQDf+8NWPcO2kwTyxdBNn/PTvLH13h99hdTjxpFOiCDjVnsgRKIzkcfOZx/HklydQGMnj8/cvZMG6rX6H1aFE40nC6noKNCUKkXYwsk85j33pFPp0LeKKhxYx940av0PqMGKaZjzwVHsi7aRHl0J+e/UpHNOzlKt/tZg/rtzid0gdgsYogs+z2jOzvmY218xWmdnrZnZ9M2UuNrMVZrbSzBaY2Wiv4hHJhm4l+Tz6xfGM6nMU1z26lMcWv+d3SL6LJzRGEXRe1l4c+JpzbjgwHphuZsOblNkATHTOjQS+B8z0MB6RrOhSGOFXV53EhCHduenxFTz04ga/Q/JVVJMCBp5nicI5t8U5tzT9vA5YDVQ1KbPAOXfgMpGXgT5exSOSTcX5Ye6/rJrJI3ryX79fxR3Pv4U7xOqHuUpdT8GXldozswHAWGBhhmJXAX9qYf+rzWyxmS2urdXNTRIMBeE87rpoHOeOq2LGs2/y7adeO+QKf7kmtcIiShQB5/ncA2ZWCjwB3OCc291CmY+SShSnNfe+c24m6W6p6urqzvlnmQRSOC/Ej88fTWVpAffOW8+qzbv55cUndJp5omLpBaQiYXU9BZmnad7MIqSSxGzn3JwWyowC7gemOOe2eRmPiB9CIeOWs4dx10XjeOP9Oj79i3/w0rrO8VVvTBRajyLQvLzqyYAHgNXOuRktlOkHzAEucc696VUsIh3Bp0b15unrJlBeFOHzDyxk0dvb/Q7JcwfWOddgdrB5meYnAJcAHzOz5enH2WZ2jZldky7zn0AF8Mv0+4s9jEfEd0N6lPHk9An06lLIf/xuZc6PWfyz60ktiiDzbIzCOTef1Nramcp8AfiCVzGIdERlhRFunTKCqx5ezAPzN3DtpMF+h+SZxkShwexAU+2J+ODjw3oyeURPfv78m7y3fa/f4XhGXU+5QYlCxCff+cwIQmb851Ov5ew9FmpR5AbVnohPjj6qiBs/eSxz19Tyl9ff9zscTxwYg1GiCDbVnoiPLj91AMN6d+HW36+iIZ7wO5x2F0+mWkqaPTbYVHsiPgrnhfjWp4axedd+Hl34rt/htLsDXU9ajyLYlChEfDZhSHdOHVzBXXPXsqch7nc47SqmrqecoNoT6QC+PnkoW+ujzFrwtt+htKuoBrNzgmpPpAMY168rnxjWk3v+vo5de2N+h9Nu4gmNUeQC1Z5IB/H1ycdS3xDn3nnr/A6l3WiMIjcoUYh0EMf16sKU0Ufz0ItvU1O33+9w2oW6nnKDak+kA/m3Tx5LLJHkl3Nzo1URU9dTTlDtiXQg/StKOG9cH37zyrvU7A5+qyKu9ShyghKFSAcz/aNDiCcdd/89+K2KxjEKrUcRaKo9kQ6mX0Ux542r4tGFwW9VRNX1lBNUeyId0HUfPYZ40nHP39f7HcoR0VKouUGJQqQD6ldRzLljq5i98J1Atyp0Z3ZuUO2JdFDXfWxI4FsVsfSkgOGQWhRBpkQh0kH1ryhpbFWseb/O73AOSyyRJJJnmClRBJkShUgH9vXJQykvinDVw4vYVt/gdzhtFosn1e2UA1SDIh1Yzy6FzLy0mpq6Bq6dvbRxIaCgSLUo9DMTdKpBkQ5uTN+juP38UbyyYXvglk2NJZ3Wy84BYb8DEJFDmzKmijc/qOOuuesY1rsLl506wO+QWkVdT7lBNSgSEF/75FA+dlwPvv/H1byzbY/f4bSKup5yg2pQJCBCIeP7U0cSyQvxH78LRhdULKGup1ygRCESIL3KC7n5zKHMX7uV3y3b5Hc4h6QWRW5QDYoEzMUn9+eE/l353jOrOvwls0oUuUE1KBIwoZDxg3NHUt8Q57Y/rPY7nIzU9ZQblChEAujYnmVcO3Ewc5Zt4uX12/wOp0VRtShygmc1aGZ9zWyuma0ys9fN7PpmypiZ3WFma81shZmN8yoekVzz5Y8OoXtpPvd24HUr4koUOcHLGowDX3PODQfGA9PNbHiTMmcBx6QfVwN3exiPSE4pjOTx+fH9mbumlrU19X6H0yx1PeUGzxKFc26Lc25p+nkdsBqoalJsCvCIS3kZOMrMensVk0iu+fz4/uSHQzz44ga/Q2mWBrNzQ1Zq0MwGAGOBhU3eqgLeO+j1Rj6cTESkBd1LC5g6poo5SzeyY0/U73A+JJpIEgkrUQSd5zVoZqXAE8ANzrndh3mMq81ssZktrq2tbd8ARQLuytMGsj+W5NFX3vU7lA+JJxwRrUUReJ4mCjOLkEoSs51zc5opsgnoe9DrPult/8I5N9M5V+2cq66srPQmWJGAGtqrjI8c052HF7zd4WaXVddTbvDyqicDHgBWO+dmtFDsaeDS9NVP44FdzrktXsUkkquuPG0gNXUNPLNis9+h/IuYup5ygpezx04ALgFWmtny9LZvAv0AnHP3AH8EzgbWAnuBKzyMRyRnTTymkiE9Srn/HxuYOraqw6woF40nyVeLIvA8SxTOuflAxm+rS81qNt2rGEQ6i1DIuGbiYL7+2Ks8/epmpozpGNeExJNO62XnAKV6kRxx7tgqRhzdhR/+6Q32xxJ+hwOo6ylXqAZFckQoZHz708PZvGs/9/9jvd/h4JxL33Cnn5mgUw2K5JDxgyqYPKInv3xhHTW79/saSyyRWi8jX3dmB54ShUiOueWsYcQSSX781zW+xhFPpi7VDatFEXiqQZEcM6B7CZedMoDHlmzktU27fIsjFk+1KNT1FHyqQZEc9JWPH0O34nwue/AV5r+11ZcYoolUi0JdT8GnRCGSg8qLIvzvl06hojSfSx5cyB3Pv0Uymd01tmPpRKEWRfCpBkVy1JAepTw5fQKfHVPFjGff5IpZi6jbH8va58fTg9kaowg+1aBIDivODzNj2mhum3o8L67dyuUPLaK+IZ6Vz442tijU9RR0ShQiOc7MuPjk/tx50ViWv7eTyx98JSvJItY4RqGfmaBTDYp0Emce35tfXDiWZe/t5MqHFrHH42ShMYrcoRoU6UTOHtmbOy4Yy5J3d3DVw4toiHs31UescYxCXU9Bp0Qh0sl8alRvfvK50by8fjvfeHyFZ1dDqespd3g5zbiIdFCfHVvFpp37uP0va+jTtYibJh/X7p/R2PWkSQEDT4lCpJP68qTBbNyxj7vmrqPqqGIuOrlfux5fYxS5Q4lCpJMyM743ZQRbdu3j20+9Ru+jCvno0B7tdvwtu1KTEpYXRdrtmOIPpXqRTiycF+LOi8ZxXK8yps9e2q5zQ/19TS1VRxUxoKK43Y4p/lCiEOnkSgvCPHT5iXQtzufyhxbx3va9R3zMaDzJi2u3MnFoZYdZllUOnxKFiNCjSyGzrjiRaDzBZQ+9ws690SM63uK3t7MnmmjXrizxjxKFiABwTM8y7ru0mo3b9/HFRxY3DkYfjhferCU/L8SpgyvaMULxixKFiDQ6eVAFPzp/FIve3sHdL6w77OPMfaOGkwZ2o6RA18vkAiUKEfkXnx1bxTmjj+aO59/i9c1tH9zetHMfb9XUM2lopQfRiR+UKETkQ/7rnBEcVZzP1x9bQTTeti6oF9bUADBJ4xM5Q4lCRD6ka0k+3596PKu37OauuWvbtO/cN2rp07WIwZUlHkUn2aZEISLNOmNEL84dW8Vdc9e2+v6KhniCBeu28tGhPXRZbA5RohCRFn3nMyOoKM1n2r0vMeOvaw65Qt6iDTvYG01ofCLHKFGISIvKiyM89qVT+dhxPbjjb2uZePsLPDh/A/tjzU9P/sKaGvLDIU7RZbE5RQPgDn4AAAdXSURBVIlCRDLqV1HMnReN4+nrJnBcrzJufWYVp/3wb/zi+bfYsSd1Y96WXft4cP4Gnly+iZMHdqM4X5fF5hJzzpu56L1SXV3tFi9e7HcYIp2Sc46X129n5rx1zF1TS1EkjyE9SlmZHsM4rlcZt009nhP6d/M5UmnKzJY456oPZ1/P0r6ZPQh8Gqhxzh3fzPvlwK+Bfuk4fuyce8ireETkyJkZpwyu4JTBFbz5QR33zVvP+q17uGnyUM46vheDKkv9DlE84FmLwsxOB+qBR1pIFN8Eyp1zN5tZJbAG6OWcyzjJjFoUIiJtdyQtCs/GKJxz84DtmYoAZZa6hq40Xdbb1d5FRKTN/BzMvhMYBmwGVgLXO+eavQXUzK42s8Vmtri2tjabMYqIdHp+JorJwHLgaGAMcKeZdWmuoHNupnOu2jlXXVmp67NFRLLJz0RxBTDHpawFNgDtv8K7iIgcET8TxbvAxwHMrCcwFFjvYzwiItIMLy+P/Q0wCehuZhuB7wARAOfcPcD3gFlmthIw4Gbn3Fav4hERkcPjWaJwzl14iPc3A2d49fkiItI+NIWHiIhkFLgpPMysFnjnMHcvB1q7ZFdrymYq09J7rd3eXLnuQLa659pyro50/2ye69Zsy9Xz3Jryh/u+znXby2b7O13inDu8y0adc53mAcxsz7KZyrT0Xmu3N1cOWNwRz1WQznVrtuXqeW5N+cN9X+e67WWD9J3ubF1Pv2/nspnKtPRea7e3JVYvHOnnd9Rz3dpt2ZLN89ya8of7vs5128sG5jsduK6nzszMFrvDnKtFWk/nOXt0rrPjSM9zZ2tRBN1MvwPoJHSes0fnOjuO6DyrRSEiIhmpRSEiIhkpUYiISEZKFCIikpESRQ4ws8+a2X1m9r9mpmlRPGRmg8zsATN73O9Yco2ZlZjZw+nv8sV+x5PL2vo9VqLwmZk9aGY1ZvZak+1nmtkaM1trZv+e6RjOuSedc18ErgH+n5fxBlk7nev1zrmrvI00d7TxnJ8LPJ7+Lp+T9WADri3nuq3fYyUK/80Czjx4g5nlAXcBZwHDgQvNbLiZjTSzZ5o8ehy067fS+0nzZtF+51paZxatPOdAH+C9dLFEFmPMFbNo/bluE89mj5XWcc7NM7MBTTafBKx1zq0HMLPfAlOccz8APt30GOl1x/8H+JNzbqm3EQdXe5xraZu2nHNgI6lksRz9EdtmbTzXq9pybFVGx1TFP/+ygtR/oKoM5b8CfAI438yu8TKwHNSmc21mFWZ2DzDWzG7xOrgc1dI5nwOcZ2Z34/8UNrmi2XPd1u+xWhQ5wDl3B3CH33F0Bs65baTGgqSdOef2kFoiWTzW1u+xWhQd0yag70Gv+6S3SfvTuc4+nfPsaZdzrUTRMS0CjjGzgWaWD1wAPO1zTLlK5zr7dM6zp13OtRKFz9Jri78EDDWzjWZ2lXMuDlwH/AVYDfyfc+51P+PMBTrX2adznj1enmtNCigiIhmpRSEiIhkpUYiISEZKFCIikpEShYiIZKREISIiGSlRiIhIRkoUkjPMrD7Ln3f/4czEeYSfeYOZFWfzM0V0H4XkDDOrd86VtuPxwukblrImPROwOeeSLbz/NlDtnNuazbikc1OLQnKamVWa2RNmtij9mJDefpKZvWRmy8xsgZkNTW+/3MyeNrO/Ac+b2SQze8HMHjezN8xsdvrHnPT26vTzejO7zcxeNbOXzaxnevvg9OuVZvbfzbV6zGxAemGZR4DXgL5mdreZLTaz183sv9LlvgocDcw1s7npbWek/x1LzewxM2u3RCnSyDmnhx458QDqm9n2KHBa+nk/YHX6eRcgnH7+CeCJ9PPLSU3F3C39ehKwi9RkaiFSUyQcON4LpP66B3DAZ9LPfwR8K/38GeDC9PNrWohxAJAExh+07cDn56U/Z1T69dtA9/Tz7sA8oCT9+mbgP/2uBz1y76FpxiXXfQIYnm4EAHRJ/9VdDjxsZseQ+pGPHLTPs8657Qe9fsU5txHAzJaT+mGf3+RzoqSSAsAS4JPp56cAn00/fxT4cQtxvuOce/mg19PM7GpSSwH0JrU62Yom+4xPb38x/e/LJ5XIRNqVEoXkuhCpv9T3H7zRzO4E5jrnpqZXBXvhoLf3NDlGw0HPEzT//ybmnHOHKJNJ42ea2UDg68CJzrkdZjYLKGxmHyOV1C5s42eJtInGKCTX/ZXUCoAAmNmY9NNy/jkv/+Uefv7LwHnp5xe0cp8upBLHrvRYx1kHvVcHlB107AlmNgTAzErM7NgjD1nkXylRSC4pTk+vfOBxI/BVoNrMVpjZKv65qtePgB+Y2TK8bVnfANxoZiuAIaTGOzJyzr0KLAPeINVd9eJBb88E/mxmc51ztaSS3G/Sx38JOK59wxfR5bEinkrf87DPOefM7AJSA9tT/I5LpC00RiHirROAO9OX1O4ErvQ5HpE2U4tCREQy0hiFiIhkpEQhIiIZKVGIiEhGShQiIpKREoWIiGSkRCEiIhn9f5Ht8Rp9TVdoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reset_seed(SEED)\n",
    "trainloader = DataLoader(trainset, batch_size=DESIRED_BATCH_SIZE, shuffle=True)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "model = ConvNet()\n",
    "model = model.to(device)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.5)\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device='cuda')\n",
    "lr_finder.range_test(trainloader, end_lr=10, num_iter=100, step_mode='exp')\n",
    "lr_finder.plot()\n",
    "lr_finder.reset()\n",
    "\n",
    "del optimizer, model, lr_finder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- With `apex.amp`, `opt_level = \"O1\"`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected optimization level O1:  Insert automatic casts around Pytorch functions and Tensor methods.\n",
      "\n",
      "Defaults for this optimization level are:\n",
      "enabled                : True\n",
      "opt_level              : O1\n",
      "cast_model_type        : None\n",
      "patch_torch_functions  : True\n",
      "keep_batchnorm_fp32    : None\n",
      "master_weights         : None\n",
      "loss_scale             : dynamic\n",
      "Processing user overrides (additional kwargs that are not None)...\n",
      "After processing overrides, optimization options are:\n",
      "enabled                : True\n",
      "opt_level              : O1\n",
      "cast_model_type        : None\n",
      "patch_torch_functions  : True\n",
      "keep_batchnorm_fp32    : None\n",
      "master_weights         : None\n",
      "loss_scale             : dynamic\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6f2adfaa5f441d59072945a78066cd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 32768.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 16384.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 8192.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 4096.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 2048.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 1024.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 512.0\n",
      "Stopping early, the loss has diverged\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEOCAYAAACD5gx6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAePUlEQVR4nO3deZRb53nf8e8DzL7PcMCdlCiSokRT+0iyItWLNluNF3mpLZ24J3bSsu5JrDo+de30pE2TxidtTprWqVunjBfZie00luwcR403RZYdyxJjLtopWSRFkUORM5jhzHAwCzADPP0DmIXDIYkZ4mK5+H3OwSFwce99H/CKP7x6ce97zd0REZHwiZS6ABERCYYCXkQkpBTwIiIhpYAXEQkpBbyISEgp4EVEQirQgDez3zKzF8zseTP7hpk1BNmeiIjMCSzgzWwd8ADQ4+47gChwX1DtiYjImYIeoqkBGs2sBmgCXg+4PRERyQks4N39OPDHwFHgBDDi7j8Iqj0RETlTTVA7NrNO4N3AJmAY+KaZfcjd/3LBejuBnQDNzc03XHHFFUGVJCISOnv37h1w99hi7wUW8MCdwKvuHgcws28BvwScEfDuvgvYBdDT0+N79uwJsCQRkXAxs9fO9V6QY/BHgTeaWZOZGXAHcCDA9kREZJ4gx+B3Aw8B+4Dncm3tCqo9ERE5U5BDNLj77wK/G2QbIiKyOF3JKiISUgp4EZGQUsCLiISUAl5EpIBOjEzQd3qy1GUACngRkYL62Nf3877P/4zx1HSpS1HAi4gUirvz0slReocm+G8/+EWpy1HAi4gUSt/pJInkNLHWer78xKs8fWy4pPUo4EVECuRgfwKAz9y7g5WtDXzqoWdJTWdKVo8CXkSkQA72jwJw7cYO/vO9O3i5b5Q/+/GhktWjgBcRKZCD8QStDTXEWuq5a/sqfvnqNXzusYOzwV9sCngRkQI51D/GlpUtZOdXhP/0zjfQWBflP/zNCyWpRwEvIlIgB+MJtsRaZl/HWut54I6tPHl4kD1HThW9HgW8iEgBjExMER9NsnllyxnL779pA51Ntfzvx4s/Fq+AFxEpgEPx7Bk083vwAE11NXzk1k089lI/B06cLmpNCngRkQKYOUVyy4IePMCv3nIpzXVRPl/kXrwCXkSkAA71J6iLRtjQ1XTWe+1NtfzKGy/hkWdf58jAWNFqUsCLiBTAwf4Em7qbiUZs0ff/xW2bqIlE+D8/OVy0mhTwIiIFcCieWHR4ZsbKtgbe37Oeh/f2Fm22SQW8iMhFmpxKc/TU+Fln0Cz0r950GdOZDF/4h+L04hXwIiIX6cjgGBlf/AfW+S5Z0cw7r1nL13Yf5dRYKvC6FPAiIhfpUH/2h9PNseYLrvubb93CxFS6KL14BbyIyEU62J/ADDbHzt+DB9i6qpV/etUavvKzIwyPB9uLV8CLiFykg/EE6zsbaaiN5rX+A7dvZSyV5os/fTXQuhTwIiIX6WB/Iq/e+4xtq1u5Z8dqHnziCCPjU4HVFVjAm9k2M3t63uO0mX08qPZEREohk3EOL5hkLB8fu30ro8lpvvREcL34wALe3V9292vd/VrgBmAc+HZQ7YmIlMLx4QmS05kLnkGz0Pa1bdy9fRVfeuJVTk8G04sv1hDNHcAhd3+tSO2JiBTF+eaguZAH7tjK6OQ0Dz5xpMBVZRUr4O8DvlGktkREimZmFsmljMHP2LGunTuvzPbiJ6fShS6NmoLvcQEzqwPeBfz2Od7fCewE2LhxY9DliIgU1MH+BCua6+hsrlvW9p++5womp9J5n4GzFMXowd8D7HP3vsXedPdd7t7j7j2xWKwI5YiIFM7B/sQFpyg4ny0rW9ixrr2AFc0pRsDfj4ZnRCSkToxMsr6zsdRlLCrQgDezZuAu4FtBtiMiUgruTjyRJNZSX+pSFhXoGLy7jwErgmxDRKRUEslpUtMZuss04HUlq4jIMg0ksnPJrGhZ3g+sQVPAi4gs00AiCaAevIhI2AzmAl49eBGRkInnhmjK9UdWBbyIyDLN9OC7lnmRU9AU8CIiyzSQSNLZVEtNtDyjtDyrEhGpAAOjqbL9gRUU8CIiyzY4lizbH1hBAS8ismwDCfXgRURCaSCRVMCLiITN5FSa0clpujVEIyISLoNj2XPg1YMXEQmZuatYFfAiIqEyNw+NhmhEREJlYFRDNCIioTQwVt4zSYICXkRkWQZGUzTXRWmsK/zNsgtFAS8isgzZq1jLt/cOCngRkWXJXuRUvj+wggJeRGRZyn2iMVDAi4gsi4ZoRERCKJ1xTo2liGmIRkQkXIbGU2S8vK9iBQW8iMiSzV3FqoAXEQmVuatYq3iIxsw6zOwhM3vJzA6Y2S1BticiUgyDY+U/0RhATcD7/yzwPXd/v5nVAU0BtyciErj4aDbgY9Ua8GbWDrwJ+DCAu6eAVFDtiYgUy0AiRW3UaGsMuo98cYIcotkExIEvm9l+M/uCmTUH2J6ISFEMJpKsaK7HzEpdynkFGfA1wPXA5939OmAM+PTClcxsp5ntMbM98Xg8wHJERApjIJGku7W8f2CFYAO+F+h199251w+RDfwzuPsud+9x955YLBZgOSIihTE4lmJFc3mPv0OAAe/uJ4FjZrYtt+gO4MWg2hMRKZaB0WTZnwMPwZ9F8zHga7kzaA4DHwm4PRGRQLk7A4lU2Z8DDwEHvLs/DfQE2YaISDGNJqdJpTMV0YPXlawiIkswkDsHvtp/ZBURCZ3BsezlPFX9I6uISBjN9uA1RCMiEi5zM0lqiEZEJFQGEinMoKtZAS8iEioDiSSdTXXURMs/Psu/QhGRMjKQSLKiAnrvoIAXEVmSwUSqIn5gBQW8iMiSDCSSrKiAH1hBAS8ikjd358TIJGvaG0pdSl4U8CIieYonkiSnM2zoqoyb0yngRUTy1Ds0AcD6zsYSV5IfBbyISJ7mAl49eBGRUOkdGgdgXYd68CIioXLs1ARdzXU015f3zbZnKOBFRPLUOzReMePvoIAXEcnb8aEJNlTI+Dso4EVE8pLJOL3DE+rBi4iEzUAiSWo6o4AXEQmbYxV2iiQo4EVE8jJziqR68CIiITNzkdM6BbyISLj0Do2zormOprrKOAceFPAiInnpHZpgfYVMMjYj0K8iMzsCjAJpYNrde4JsT0QkKL1DE2xf21bqMpakGP+v8VZ3HyhCOyIigchknONDE9z9hlWlLmVJNEQjInIB8USSVDpTUadIQp4Bb2abzaw+9/wtZvaAmXXksakDPzCzvWa282IKFREplWOnKu8USci/B/8wkDazLcAuYAPw9Ty2u83drwfuAX7DzN60cAUz22lme8xsTzwez7duEZGimTlFckNIAz7j7tPAe4D/6e6fBNZcaCN3P577sx/4NnDTIuvscvced++JxWL5Vy4iUiRzFzmFcIgGmDKz+4FfBR7JLas93wZm1mxmrTPPgbuB55dbqIhIqfQOTdDdUk9DbbTUpSxJvmfRfAT4KPAZd3/VzDYBf3GBbVYB3zazmXa+7u7fW3alIiIl0jtUWbNIzsgr4N39ReABADPrBFrd/b9eYJvDwDUXXaGISIn1Do2zY117qctYsnzPonnczNrMrAvYB/y5mf1JsKWJiJReOuMcH56ouPF3yH8Mvt3dTwPvBb7q7jcDdwZXlohIeegfnWQq7RU5RJNvwNeY2RrgA8z9yCoiEnqzp0hW2Dw0kH/A/z7wfeCQu//czC4DXgmuLBGR8lCJ88DPyPdH1m8C35z3+jDwvqCKEhEpF72ncvPAd1RewOf7I+t6M/u2mfXnHg+b2fqgixMRKbXeoQlirZV3DjzkP0TzZeA7wNrc429zy0REQu3Y0HhFDs9A/gEfc/cvu/t07vEgoHkFRCT0shc5Vd4PrJB/wA+a2YfMLJp7fAgYDLIwEZFSc3dOjkyytqOh1KUsS74B/2tkT5E8CZwA3g98OKCaRETKwumJaVLpDLGW+lKXsix5Bby7v+bu73L3mLuvdPd70Vk0IhJy8cQkALHWEAf8OXyiYFWIiJSh+GgKINw9+HOwglUhIlKG4okkAN1V2IP3glUhIlKGBkazAV+pPfjzXslqZqMsHuQGVOaJoSIieYonktREjPbG897fqGydN+DdvbVYhYiIlJuB0STdLfVEIpU5In0xQzQiIqEWTyTpbq0rdRnLpoAXETmHgUSyYsffQQEvInJO8dwQTaVSwIuILCKTcQYTqYq9yAkU8CIiixqemGI64+rBi4iEzUDuIif14EVEQiaeu8hJPXgRkZBRDz4Pufnj95vZI0G3JSJSKPEKn6YAitOD/zfAgSK0IyJSMPFEkrpohLbG817wX9YCDfjcjbl/GfhCkO2IiBRa9hz4Oswqc5oCCL4H/z+AfwdkAm5HRKSgBir8HHgIMODN7B1Av7vvvcB6O81sj5nticfjQZUjIrIkAxV+FSsE24O/FXiXmR0B/gq43cz+cuFK7r7L3XvcvScWiwVYjohI/uKJpHrw5+Luv+3u6939UuA+4DF3/1BQ7YmIFEo645waS6kHLyISNkPjKdIZr/gefFHO/3H3x4HHi9GWiMjFmrnIST14EZGQmZumoHJv9gEKeBGRs4RhmgJQwIuInGW2B6+AFxEJl4FEivqaCK31lTtNASjgRUTOMnOrvkqepgAU8CIiZxkIwUVOoIAXETlLpd9se4YCXkRkAfXgRURCaDqdYXAsRazCz4EHBbyIyBlOjadwr/xz4EEBLyJyhjDcbHuGAl5EZJ6BRApQD15EJHTUgxcRCamwzEMDCngRkTPER5M01kZprvBpCkABLyJyhrCcAw8KeBGRM2SvYq38c+BBAS8icgb14EVEQios89CAAl5EZNbweIqh8SlWtzWUupSCUMCLiOQ8/nIcgNu2dpe4ksJQwIuI5PzwQB/dLfVcs76j1KUUhAJeRARITWf48ctx7rxyJZFIZd/JaYYCXkQE2P3qIInkNHdeuarUpRRMYAFvZg1m9o9m9oyZvWBmvxdUWyIiF+vRF/toqI1w65ZwjL8DBHktbhK43d0TZlYL/NTMvuvuTwXYpojIkrk7jx7o57YtMRrroqUup2AC68F7ViL3sjb38KDaExFZrgMnRjk+PMFd21eWupSCCnQM3syiZvY00A/80N13B9meiMhyPHqgDzO4/YrwjL9DwAHv7ml3vxZYD9xkZjsWrmNmO81sj5nticfjQZYjIrKoRw/0ce2GjtBMUTCjKGfRuPsw8CPg7Yu8t8vde9y9JxaLFaMcEZFZJ0cmebZ3JFRnz8wI8iyamJl15J43AncBLwXVnojIcvz9S30A3LU9fAEf5Fk0a4CvmFmU7BfJX7v7IwG2JyKyZI++2MfGria2rmwpdSkFF1jAu/uzwHVB7V9E5GJNpNI8cWiQD918CWbhuHp1Pl3JKiJVa9/RIVLTGf7J5eG5uGk+BbyIVK3dhweJGPRc0lnqUgKhgBeRqvXUq6fYsa6d1obaUpcSCAW8iFSlyak0Tx8b5uZNXaUuJTAKeBGpSvuPDpOaznDzphWlLiUwCngRqUq7Xx3EDG5UD15EJFx2Hz7FlavbaG8M5/g7KOBFpAolp9PsOzrEzZeFt/cOCngRqULP9o6QDPn4OyjgRaQK7T48CMBNIR5/BwW8iFSh3a+eYtuqVrqa60pdSqAU8CJSVabSGfa+Fv7xd1DAi0iVee74COOpdOjH30EBLyJVZvfhU0D4x99BAS8iVWb3q4NsjjWH7vZ8i1HAi0jVmE5n2HNkiJsvC//wDCjgRaSKvHRylERyOtQTjM0X5C37iuZTDz1LKp0BYP49WRxwd3zmtc8tJ/feQmaGAWaQ8dz2Z6/GWY3lYeHq89syIJJ7Epm3/MztjUgk+yxii7/PvP3B3GddrI7s3w/4omvNmf/5IzbTtmEGUTOiEcPMiEZyNeaKj9jc+pGIETGbXT8aMSKR7OuZ9Sz3Z3Z/UBOJEI1ANBKhZmYbm9n/3LrRyJn7rcnte2abumiEmmiE2qhRG41QF40QiYTv7j1yYfuPDgFw/cZwzv++UCgCfv+xISanMrNB5T4XfoadEXgzt+Wa/ee94Bth/pfCTOjYvO1mVz1n6i/urLUXtDUTtJns9xSZRfbvnl0+88Wz2P7n72/u72Cx9Tjjsy0Wd878v7eZGnM1ZLJffGn33Ovs8/ntl7NoxM4I/JqZ5zUR6mui1NdEqK+J0FAbpbE2SmNdlIbaKM11UVoaamipzz0aamhtqKUt92dHUy2dTXVE9QVSlvYfHaa7pZ71nY2lLqUoQhHwP/itN5e6BFlEJpMN+3Qm9yXgnn2egelMJveFMPellcnMW8ez280+3JlOZ7JfMJm5L7m0z60zncl+8cwsm05n/5zKZJhOO1PpDFNpJzWdYSqdIZXOkJrOMJ3JMDWdfT+ZW5aczjA5lWZ4YoqTI5NMTKUZT6WZSE0zlkqf93NHDLqa6+huqSfWWs+qtgZWtzWwqr2BdR0NbOpuYUNnIzVRjZAW276jQ1y/sSOU919dTCgCXsrTzDBI2Hqz6YwzlppmLDnN6YlpRienGJ2c5vTkFMPjUwwkkgwkksRHU8RHJ3mlL0H/6CSZef9XUxs1NnY1cVmshcu6m7ks1sym7hYu7W4i1lJfNQFUTKfGUhwZHOeDN24sdSlFo4AXWaJoxGhrqKWtoZY17fltk844A4kkvUMTHI4nODwwlv0zPsaPX47P/oYEUF8TYV1HI+s6G9nY1cSNl3Zxy+YVrGprCOgTVYe58feOEldSPAp4kSKIRoxVbQ2samvghgU3eE5nnNeHJzg8MMaRgTGOD09wfGiC3uEJvvPM63xt91EANsea+aXN3dx+5UpuuWwFDbXRUnyUirX/6DDRiHHV+jy/lUNAAS9SYtGIsaGriQ1dTbz58tgZ76UzzoETp/nZoQF+dmiQh/b28hdPvUZTXZQ3bY1x5/ZV3HHFSjpDPmlWIew7OsSVa1ppqque2Avsk5rZBuCrwCqyJ1/scvfPBtWeSBhFI8aOde3sWNfOzjdtZnIqzZOHB3n0xT4ePdDH9144STRi3Lypi7e9YTV3v2EVa9qr4wyRpUhnnGeODfPe69eXupSisqWe7pf3js3WAGvcfZ+ZtQJ7gXvd/cVzbdPT0+N79uwJpB6RsHF3njs+wvdfOMn3X+jjYH8CgHUdjVy5po3ta9vYvqaNGy7prIrL8s/nwInT3PPZf+C/f/Aa3nNduELezPa6e89i7wXWg3f3E8CJ3PNRMzsArAPOGfAikj8z4+r1HVy9voNPvu0KDvYneOylPp4/fpoXT5zmsZf6Zs/c2bqyhVs2r+CWy1Zw29ZuWhvCex/Sxew/OgzAdRuq4wKnGUUZjDKzS4HrgN3FaE+kGm1Z2cKWlS2zrydSaQ6cPM3uw6d48nB2/P6rT75GXU2EN18e4x1Xr+GOK1fRUh/+Mel9R4foaq7jkhVNpS6lqAI/smbWAjwMfNzdTy/y/k5gJ8DGjdVzfqpI0Brroly/sZPrN3byr9+ymal0hv1Hh/ne8yf5u+dO8MMX+6iridBzSSc3beripku7uG5jJ4114Ts7Z//RIa7bUD0XOM0IbAwewMxqgUeA77v7n1xofY3BixRHJuPsOzrEd58/yZOHBjlw8jTuUBPJXoC1pqOBte2NrO1oZMvKFq5e387GrqaKDMjh8RTX/v4P+eTbtvEbb91S6nIKriRj8Jb9L+GLwIF8wl1EiicSMXou7aLn0uysiqcnp9j72hB7jpziyMA4r49M8JNX4vSPJmfnFWprqOGq9e3sWNvOlWvauHJNG5fFmqkt8ykXnj42M/5ePRc4zQhyiOZW4J8Dz5nZ07ll/97d/y7ANkVkGdoaannrtpW8ddvKM5Ynp9O80pfgueMjPNs7wnPHh/nyE0dmr7yti0bYvLKFzbHm2d8ArljdxuZYc9n09vcdHSZicLUCvnDc/acseUJdESkn9TXR2fPw778pu2wqneFwfIwDJ05z4MRpftE3yjO9w/y/507M9vY7mmq5fmMnN1zSyXUbO7hqXXvJztzZf3SIy1e1VsWPyQtV3ycWkYtSG42wbXUr21a3cu9162aXT06lORwf4/njI+x9bYi9R4d47KV+IDvd9OZYC9es7+DKNa2sbs/NsNnWQKy1PrBpF9IZ5+ljw7zzmrWB7L/cKeBFpCAaaqPZi6vWtvGBGzcA2R84n+kd4ZljwzxzbJgf/6Kfh/f1nrVtc12UzuY6unKPFc31dLfW0d1cz4qWOla01LNi5r2WOuprzv5CGBmf4vnXR3j++Aiv9Cd4pW+Ug/0JxlJpbqiSG3wspIAXkcB0NNXx5stjs3PsuDvD41P0jU7SdzpJ3+lJ4qNJTo2lZh8DiSS/ODnKQCJ1xiyb87U31rKytZ6VbfU019Xwct8orw2Oz76/srWerata+Gc9G9i2upV3XLOmKJ+33CjgRaRozIzO5jo6m+u4YvX513V3EslpBhIpTo0lGUykGBxLMTCaJJ5I0n86OftFsX1NGx+8cQNXrcue5aPJ17IU8CJSlsyM1oZaWhtq2dTdXOpyKlJ5n8AqIiLLpoAXEQkpBbyISEgp4EVEQkoBLyISUgp4EZGQUsCLiISUAl5EJKQCveHHUplZHHhtmZu3AyMBrH+h9c73/rneW2z5Ysu6gYE8aiy0pf5dFnI/+WxTjccECnNcgjom+awX1HGp9GOy3P3MbHOJu8cWXcPdQ/EAdgWx/oXWO9/753pvseXnWLanEv4uC7mffLapxmNSqOMS1DEp5XGp9GMS5HEJ0xDN3wa0/oXWO9/753pvseVLrT9IhaplOfvJZ5tqPCZQmHqCOib5rBfG41LW/1bKaohGzmRme/wc91qU0tAxKT86JucWph58GO0qdQFyFh2T8qNjcg7qwYuIhJR68CIiIaWAFxEJKQW8iEhIKeArkJnda2Z/bmb/18zuLnU9kmVml5nZF83soVLXUs3MrNnMvpL7N/Irpa6nlBTwRWZmXzKzfjN7fsHyt5vZy2Z20Mw+fb59uPvfuPu/BD4KfDDIeqtFgY7LYXf/9WArrU5LPD7vBR7K/Rt5V9GLLSMK+OJ7EHj7/AVmFgX+F3APsB2438y2m9lVZvbIgsfKeZv+Tm47uXgPUrjjIoX3IHkeH2A9cCy3WrqINZYd3XS7yNz9J2Z26YLFNwEH3f0wgJn9FfBud/9D4B0L92FmBvwX4Lvuvi/YiqtDIY6LBGcpxwfoJRvyT1Plndiq/vBlZB1zPQ7I/ge67jzrfwy4E3i/mX00yMKq3JKOi5mtMLM/A64zs98Oujg55/H5FvA+M/s85TWtQdGpB1+B3P1PgT8tdR1yJncfJPu7iJSQu48BHyl1HeVAPfjycBzYMO/1+twyKS0dl/Km43MBCvjy8HNgq5ltMrM64D7gOyWuSXRcyp2OzwUo4IvMzL4BPAlsM7NeM/t1d58GfhP4PnAA+Gt3f6GUdVYbHZfypuOzPJpsTEQkpNSDFxEJKQW8iEhIKeBFREJKAS8iElIKeBGRkFLAi4iElAJeyp6ZJYrc3hdysxIWs82Pm1lTMduU8NN58FL2zCzh7i0F3F9N7iKZosnNAGrunjnH+0eAHncfKGZdEm7qwUtFMrOYmT1sZj/PPW7NLb/JzJ40s/1m9jMz25Zb/mEz+46ZPQb8vZm9xcweN7OHzOwlM/taLoTJLe/JPU+Y2WfM7Bkze8rMVuWWb869fs7M/mCx/8sws0tzN6P4KvA8sMHMPm9me8zsBTP7vdx6DwBrgR+Z2Y9yy+7OfY59ZvZNMyvYF5xUEXfXQ4+yfgCJRZZ9Hbgt93wjcCD3vA2oyT2/E3g49/zDZKeT7cq9fgswQnaCqgjZy+Bn9vc42d40gAPvzD3/I+B3cs8fAe7PPf/oOWq8FMgAb5y3bKb9aK6dq3OvjwDduefdwE+A5tzrTwH/sdTHQY/Ke2i6YKlUdwLbc51ugLZcL7cd+IqZbSUbzrXztvmhu5+a9/of3b0XwMyeJhvIP13QTopsmAPsBe7KPb8FuDf3/OvAH5+jztfc/al5rz9gZjvJTtW9huydiJ5dsM0bc8ufyH2+OrJfQCJLooCXShUh2zOenL/QzD4H/Mjd35O7A9Dj894eW7CP5LznaRb/9zDl7n6Bdc5ntk0z2wT8W+BGdx8ysweBhkW2MbJfRvcvsS2RM2gMXirVD8je2QoAM7s297SduTnBPxxg+08B78s9vy/PbdrIBv5Ibiz/nnnvjQKt8/Z9q5ltATCzZjO7/OJLlmqjgJdK0JSbInbm8QngAaDHzJ41sxeZu5PSHwF/aGb7Cfb/UD8OfMLMngW2kB3PPy93fwbYD7xEdljniXlv7wK+Z2Y/cvc42S+nb+T2/yRwRWHLl2qg0yRFliF3zvqEu7uZ3Uf2B9d3l7oukfk0Bi+yPDcAn8udWjkM/FqJ6xE5i3rwIiIhpTF4EZGQUsCLiISUAl5EJKQU8CIiIaWAFxEJKQW8iEhI/X9gTeHNQCFArQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reset_seed(SEED)\n",
    "trainloader = DataLoader(trainset, batch_size=DESIRED_BATCH_SIZE, shuffle=True)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "model = ConvNet()\n",
    "model = model.to(device)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.5)\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "model, optimizer = amp.initialize(model, optimizer, opt_level='O1')\n",
    "\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device='cuda')\n",
    "lr_finder.range_test(trainloader, end_lr=10, num_iter=100, step_mode='exp')\n",
    "lr_finder.plot()\n",
    "lr_finder.reset()\n",
    "\n",
    "del optimizer, model, lr_finder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- With `apex.amp`, `opt_level = \"O2\"`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected optimization level O2:  FP16 training with FP32 batchnorm and FP32 master weights.\n",
      "\n",
      "Defaults for this optimization level are:\n",
      "enabled                : True\n",
      "opt_level              : O2\n",
      "cast_model_type        : torch.float16\n",
      "patch_torch_functions  : False\n",
      "keep_batchnorm_fp32    : True\n",
      "master_weights         : True\n",
      "loss_scale             : dynamic\n",
      "Processing user overrides (additional kwargs that are not None)...\n",
      "After processing overrides, optimization options are:\n",
      "enabled                : True\n",
      "opt_level              : O2\n",
      "cast_model_type        : torch.float16\n",
      "patch_torch_functions  : False\n",
      "keep_batchnorm_fp32    : True\n",
      "master_weights         : True\n",
      "loss_scale             : dynamic\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90880c13b2224171b25396994eaf110a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 32768.0\n",
      "Gradient overflow.  Skipping step, loss scaler 0 reducing loss scale to 16384.0\n",
      "\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deZxcZZ3v8c+vqnrft3Q6naXTWWlCwpJASJBFIIigoLjACAroMHhRZNS5zoz3er3XccZxRueqODo4IHoHxYVFXBAQEAgkkJCEhCyQpLN2tt73rbqe+0dVQgjdle5OnzrV1d/361WvVNc5VeeXk05/+znPc57HnHOIiIgMJeB3ASIiktwUFCIiEpeCQkRE4lJQiIhIXAoKERGJS0EhIiJxhfwuYKRKS0tdVVWV32WIiIwrr776aoNzrmw07x13QVFVVcXatWv9LkNEZFwxsz2jfa8uPYmISFwKChERiUtBISIicSkoREQkLgWFiIjEpaAQEZG4PBsea2bTgJ8C5YAD7nHOfeeEfa4BvgZEgDBwl3NupRf1OOcwsyG39fRH6I9ECA84wpEI/QOOvnCE3vAAvf0RggEjLzNEbkaIrPQg7T1hmjr7aO7so7NvgFDACMYeQxwGAMMIGAQCsT8t+p6AGWnBALmxY+RmhAgG4nyQiEiCeHkfRRj4gnNunZnlAa+a2VPOuS3H7fM08JhzzpnZQuCXwHwvinn89UN85mfryAgFSQ8FSA8FcA66+8J09Q+QjMty5KQHKchKIz8rjbzM0LEgCpiREQpQnJNOcU4GJTnpZKQFMIuGTyhgZISCZIQCZKYFMYP+AUf/QIT+gQhpwQCZaQEyQtFtHT1hOnqjj/CAI2BEQ8qMgYEI4YgjHHGEAkZpbgaluRmU5KYzpTCLgqw0v0+TiHjMs6Bwzh0EDsaet5vZVqAS2HLcPh3HvSWHaMvDE7PKcrnjktn0hiPHWgpgZKcHyUkPkpkeJD0YIBQwQrE/M9NioRIMEI44OmM/TLv7B8jLDFGcnU5RTjrZ6UEiDgZiLZJ4f4mIc+CI7u8ckYhjIOIYcNEf5J29Ydp73nq09fTT1t1PW08/kQiEByJEnKOhI8KmulaaOvvoH/Av5fIzQ0wrzmZqURbFOekUZKVTlJ1GSW4GFQWZTC7IpKIgk+z0cXdvp4jEJOR/r5lVAWcBLw+y7QPAPwGTgKuGeP9twG0A06dPH1UN8ybnMW/yvFG9N5k552jvDdMfjhBx0SAKR6KXzXr6B+gNRxiIODJCAdKCAYIBIxyJ0Nsf3R5xHLuklpsZIi0QiAZY7BEKRN+TFjT6whEaO/toaO+loaOPAy3d7G3qYl9zF7saOlm3t4WWrsGDa3J+JnPKc5kzKY+ZZTlMzs+kPD+D8vxMSnMzdJlNJImZ10uhmlku8Bzwdefcw3H2uxD4inPusnift3jxYqcpPJKXc46uvgEaOno52NrDodYe6lq62VnfwfbDHew40kF3/8Db3hMKGFMKs5halMX04mxOq8hnQWU+NRUFZKUHffqbiKQWM3vVObd4NO/1tEVhZmnAQ8AD8UICwDn3vJlVm1mpc67By7rEO2ZGTkaInIwQM0py3rE9EnEcae/lSHsPh9t6OdzWw4GWbvY3d7O/uYsntxzmwTX7AAgYVBZlURLrhynJTWdGSQ5zJuUypzyP6cXZaolISmvt6mfN7ibW7G7ild1NXLNoCjcvn5nwOrwc9WTAvcBW59y3h9hnNrAz1pl9NpABNHpVk/gvEDAmx/ouBuOc41BbD6/XtbGprpU9jZ00dfZxsLWHTXWtHGnff2zfjFCAOeW5zC3PY155HmdMLeDs6UVkpqkVIsmnNzzA4dZeDrf30NLVT0tXH63d/dHRk119sT/7ae8J09HbT0dPmJbufpyDtKCxcGoh+T4NHvGyRbEcuAnYZGYbYq/9PTAdwDn3Q+A64ONm1g90Ax91Xl8Lk6RmZlQUZFFRkMXlNeXv2N7RG2b74Xa2H+7gjcPtvHm4nRd3NPDwujoA0kMBzp5eyPnVpVw8r4wzKgsIqNUhCeScY39zN+v2NrNuTzMb9reyv6mLxs6+QfcPBoyi7HRKctIpzE6jsjCLvMw88jJDlOdncs6MIs6cVujrL0Ce91GMNfVRyGBauvpYt7eZl3Y0sqq2kS0H23AOyvIyuHT+JN49fxLLZpeSm6HRVzI2nIuOWAxHHM1dfayubeTFHY28tKOBA609AGSnB1k4tYCZpblMibWky/MzKcqOhkJ+Vhr5maEh7/EaS6fSR6GgkJTU1NnHn984wtNbj/Dcm/V09IYJBYxzZhRx4dwyrlwwmeqyXL/LlCTmnONwWy+1DR3sauhkd0Mne5u62NPYxf7mbjp6w+94T2F2GstmlbC0uoSzpxcxf3IeoWByTIChoBCJoy8cYe2eJp5/s4Hn36xny8E2AJZUFfGRxdO4amGF7vMQ+sIRNu5v4eVdTby8q4l1e5rfFgbpoQDTi7OPPfIzQ9F7roJGTnqIc2YUUVORn7SXOhUUIiNwuK2HR9bX8cs1+6ht6CQ3I8SNS2fwqXfNpDQ3w+/yxCc3/ufLrNwRHXA5rzyPxVXRFsHM0lxmluVQkZ+ZtCEwHAoKkVFwzrF2TzM/eWk3v990kIxQgI+dN4O/urCaSfmDj8qS1LXwq0+wfHYpX//AGRTnpPtdzphL2vsoRJKZmbGkqpglVcX8dX0H3392B/e/tJtfrNnHl686jeuXTEtIJ6P4r62nn7aeMGdOK0zJkDhVydHLIuKzWWW5fPsjZ/L05y/ijMoC/u7hTXzix2s40NLtd2mSAHXN0X/nyqIsnytJTgoKkeNUlebwwKfO42vXnM6aXU1c8W/P8+j6Or/LEo8dC4pCBcVgFBQiJwgEjJvOr+KJuy5k3uQ87vrFBr7064109w2c/M0yLtXFWo5Ti7J9riQ5KShEhjC9JJsHb1vKZy6ZzS9f3cc131/J9sPtfpclHqhr6SYjFKA0V/0Tg1FQiMQRCgb44hXz+Omt59LU2cc133+RdXub/S5LxlhdczeVhVkavDAEBYXIMLxrThm/v/NdlOVl8Mn717CzvuPkb5JxY39zlzqy41BQiAxTeX4mP731XIIB4+P3vsLhth6/S5IxUtfSrY7sOBQUIiMwoySHH998Ls1dfdz84zW09fT7XZKcop7+ARo6+piqFsWQFBQiI3TG1AJ+eOM5bD/czh0PrGMgMr5mN5C3OzriSZeehqagEBmFC+eW8bVrF/DC9ga++/R2v8uRU/DWPRQaGjsUBYXIKF2/ZBrXnT2V7z6zneferPe7HBml/bor+6QUFCKjZGb8w7ULmFeex10Prtd0H+NUXUsXoYBRnqeZg4eioBA5BVnpQf79Y2fTP+C442fr6AtH/C5JRqiuuZvJBZlJs8BQMtKZETlF1WW5/PN1C1m/t4X7XtzldzkyQhoae3IKCpExcNXCCi6aW8YP/rxTQ2bHmbrmbvVPnISCQmSM/M0V82jt7udHz9f6XYoMU/9AhENtPUxViyIuBYXIGFlQWcBVZ1Rw78pdNHT0+l2ODMOh1h4iTrPGnoyCQmQMfX7FXHrDEb7/7A6/S5Fh0NDY4VFQiIyhWWW5fOjsqTyweu+xO34leR27K1uXnuJSUIiMsTsvmwPAd/70ps+VyMkcvSu7ojDT50qSm4JCZIxVFmZx49IZ/PrV/VroKMntb+5iUl4GGaGg36UkNQWFiAfuuGQW2ekhvvnEG36XInHUtXRr1thhUFCIeKAkN4PbL6rmqS2HWbu7ye9yZAh1Ld1UasTTSXkWFGY2zcyeNbMtZrbZzD43yD4fM7ONZrbJzF4ys0Ve1SOSaLdeMJNJeRn84x+24pymIk82kYjjYEuPOrKHwcsWRRj4gnOuBlgK3GFmNSfsswu4yDl3BvA14B4P6xFJqOz0EH99+VzW7W3hic2H/S5HTlDf0UvfQERDY4fBs6Bwzh10zq2LPW8HtgKVJ+zzknPu6Er1q4GpXtUj4ocPnzOVWWU5fPOJbYQHNGFgMtnf3AWgPophSEgfhZlVAWcBL8fZ7ZPA40O8/zYzW2tma+vrNe+/jB+hYID//p751NZ38uCafX6XI8c5erOdpu84Oc+DwsxygYeAu5xzbUPscwnRoPjSYNudc/c45xY75xaXlZV5V6yIB1bUlHPezGL++Y/bONzW43c5EqO7sofP06AwszSiIfGAc+7hIfZZCPwncI1zrtHLekT8YGZ847qF9IUjfPmR19WxnSRq6zuZnJ9JdnrI71KSnpejngy4F9jqnPv2EPtMBx4GbnLO6TZWSVkzS3P44op5/GnrYX678aDf5QhQ29DBzNIcv8sYF7xsUSwHbgLebWYbYo/3mtntZnZ7bJ+vACXAv8e2r/WwHhFf3XrBTBZNK+Srj22mUbPL+so5R219J9VlCorh8KzN5ZxbCdhJ9vkU8CmvahBJJsGA8S8fWshV332Br/52C9+74Sy/S5qwmrv6ae3up7os1+9SxgXdmS2SQHPL8/jsu+fw29cOsG5v88nfIJ6ore8AUItimBQUIgl28/Iq0oLGE68f8ruUCau2vhOAavVRDIuCQiTB8jPTOH9WKU9sPqQRUD6pbegkPRjQynbDpKAQ8cGKmnJ2N3ax/UiH36VMSLX1HcwoySYYiNuNKjEKChEfXF5TDsCTm3X5yQ+1DZ0aGjsCCgoRH5TnZ3LmtEKe3KLJAhMtPBBhT2OnRjyNgIJCxCcrTi9n4/5WDmht7YSqa+mmf8BpxNMIKChEfLKiZjIAf9qqVkUiHR3xNEtBMWwKChGfzJ6US3VZDk9qrYqE2hm7h2JmqS49DZeCQsRHK2oms7q2kdaufr9LmTB2NXRSmJ1GcU6636WMGwoKER+tOL2ccMTx7BtH/C5lwqit79SNdiOkoBDx0ZlTCynLy+AJDZNNmOissbrsNBIKChEfBQLGpfMnsXJ7g5ZKTYDO3jCH23o14mmEFBQiPls+u5T23jAb61r9LiXl7WrQiKfRUFCI+GzZrBIAXtrR4HMlqU8jnkZHQSHis5LcDE6ryOfFHVoJ2Gu19Z2YwYwSTQY4EgoKkSRwwewSXt3TTHffgN+lpLRdDZ1MLcoiMy3odynjioJCJAksm11K30CEtXua/C4lpdU2dFCty04jpqAQSQLnVhUTCpguP3nIOceues0aOxoKCpEkkJMR4uzpRby0Ux3aXnDO8bXfbaWzb4CFUwv8LmfcUVCIJIlls0vYVNdKS1ef36WklIGI428f2sR9L+7iluVVXHtmpd8ljTsKCpEkccHsUpyD1bW6/DRW+sIR7nxwPb9Yu487L53DV66uIaBV7UZMQSGSJBZNKyQnPah+ijHinOOLv3qN3288yJffexqfv3wuZgqJ0VBQiCSJtGCA86pLeFE33o2JRzfU8dhrB/jC5XP5ywur/S5nXFNQiCSRZbNKqG3o5GCrVr07FQdauvnKbzazeEYR/+2S2X6XM+4pKESSyAVzSgF0+ekURCLRS04DEce3PrKIoPokTpmCQiSJzCvPoyQnnVU7FRSj9ZNVu3lpZyP/8+oaZpTonomxoKAQSSJmxnnVxRr5NErbD7fzjce38e75k7h+yTS/y0kZngWFmU0zs2fNbIuZbTazzw2yz3wzW2VmvWb2Ra9qERlPllaXUNfSzb6mLr9LGVe6+sJ8+oF15GWG+MZ1Z2iE0xjyskURBr7gnKsBlgJ3mFnNCfs0AXcC/+phHSLjytLq6LTjq9SqGDbnHF9+5HV21nfwnevPYlJept8lpRTPgsI5d9A5ty72vB3YClSesM8R59waQCvLi8TMmZRLSU66Lj+NwINr9vHI+jruunQuy2eX+l1OyklIH4WZVQFnAS+P8v23mdlaM1tbX18/lqWJJB0zY2l1CS/XNuGc87ucpLf5QCv/67HNvGtOKZ99t4bCesHzoDCzXOAh4C7nXNtoPsM5d49zbrFzbnFZWdnYFiiShJZWF1PX0s3+Zt1PcTJf+OVrFGen838/eqam5/CIp0FhZmlEQ+IB59zDXh5LJJWon2J4nHO8ebid686ppCQ3w+9yUpaXo54MuBfY6pz7tlfHEUlFsyflUpqbzmrdTxFXZ98AEQf5mWl+l5LSQh5+9nLgJmCTmW2Ivfb3wHQA59wPzWwysBbIByJmdhdQM9pLVCKpIno/RQmraxtxzmmo5xDae6LjYPKzFBRe8iwonHMrgbjf3c65Q8BUr2oQGc+WVpfw+40H2dfUzfSSbL/LSUpt3WEA8jK9/J1XdGe2SJI6v7oY0PoU8RxrUejSk6cUFCJJalZZtJ9CHdpDa4sFhVoU3lJQiCSpE/sp5J3ae6KXntRH4S0FhUgSWz6rlIOtPWzc3+p3KUmprUd9FImgoBBJYu9bVEFuRogfv7jL71KSUlu3+igSQUEhksTyMtP48OKp/G7jQQ639fhdTtJp7wmTHgyQEdKPMi/p7IokuVuWzWTAOX66arffpSSdtp5+8jJDus/EYwoKkSQ3vSSby08r52cv76W7b8DvcpJKe09YHdkJoKAQGQduvWAmzV39PLqhzu9Skkpbd786shNgWEFhZrPMLCP2/GIzu9PMCr0tTUSOOm9mMTUV+dy3cpeGyh6nvadfHdkJMNwWxUPAgJnNBu4BpgE/86wqEXkbM+OTF8xk+5EOXtje4Hc5SaOtJ6wWRQIMNygizrkw8AHge865vwEqvCtLRE509aIKSnMz+OmqPX6XkjTUokiM4QZFv5ndAHwC+F3sNf3riCRQRijI+xdN4fk364/NcTTRtatFkRDDDYpbgPOBrzvndpnZTOD/eVeWiAzmvWdMpm8gwjPbjvhdiu/6ByJ09Q2QpxaF54YVFM65Lc65O51zPzezIiDPOffPHtcmIic4e3oRk/Iy+OPrh/wuxXcdx+Z5UovCa8Md9fRnM8s3s2JgHfAjM9OqdSIJFggYV5w+mWffOEJXX9jvcnz11syxalF4bbiXngpiq859EPipc+484DLvyhKRoVy5YDI9/RGee6Pe71J8dWzmWPVReG64QREyswrgI7zVmS0iPjh3ZjHFOek8PsEvPx2dEFAtCu8NNyj+D/AEsNM5t8bMqoHt3pUlIkMJBQOsqCnn6a2H6emfuFN6tKmPImGG25n9K+fcQufcp2Nf1zrnrvO2NBEZynsWTKazb4CVE/jmOy2DmjjD7cyeamaPmNmR2OMhM5vqdXEiMrhls0rJzwxN6MtPx1oUCgrPDffS04+Bx4ApscdvY6+JiA/SQwEuqynnqS2H6AtH/C7HF0dbFLnqzPbccIOizDn3Y+dcOPa4HyjzsC4ROYkrF1TQ1hPm5V2Nfpfii7buMDnpQYIBrUXhteEGRaOZ3WhmwdjjRmBifneKJInls0sIBYxVOyfmf8X2nn6tRZEgww2KW4kOjT0EHAQ+BNzsUU0iMgzZ6SEWVBawZneT36X44ujqduK94Y562uOce79zrsw5N8k5dy2gUU8iPjt3ZjGv7WudkMNk23vC6shOkFNZ4e7zY1aFiIzKkqpi+gYibNzf6ncpCacWReKcSlCoB0nEZ4tnFAHwygTs0NZ62YlzKkERdz1GM5tmZs+a2RYz22xmnxtkHzOz75rZDjPbaGZnn0I9IhNOUU46c8tzeWV3s9+lJJzWokicuGfZzNoZPBAMyDrJZ4eBLzjn1plZHvCqmT3lnNty3D5XAnNij/OAH8T+FJFhWlJVzG82HGAg4ibMUFHnHG3dWt0uUeK2KJxzec65/EEeec65uCHjnDvonFsXe94ObAUqT9jtGqKz0Trn3GqgMDb5oIgM07kzi+noDbP1YJvfpSRMT3+EcMRpQsAEOZVLT8NmZlXAWcDLJ2yqBPYd9/V+3hkmmNltZrbWzNbW10/sqZVFTnTuzGIAXtk1cYbJvrUWhS49JYLnQWFmucBDwF2xNS1GzDl3j3NusXNucVmZbggXOV5FQRZTi7Im1P0UxyYEVGd2QngaFGaWRjQkHnDOPTzILnXAtOO+nhp7TURG4NyqYtbsbsK5uGNMUkZrd3RCQLUoEsOzoDAzA+4Ftjrnhlo29THg47HRT0uBVufcQa9qEklVS2YW09DRx66GTr9LSQhNMZ5YXsbxcuAmYJOZbYi99vfAdADn3A+BPwDvBXYAXcAtHtYjkrKWVL3VT1FdlutzNd7TMqiJ5dlZds6t5CQ35bloO/kOr2oQmShmleVQkpPOK7ubuP7c6X6X47k29VEkVEJGPYmIt8yMJVXFE2bk09EWhfooEkNBIZIils8uYX9zNzuOdPhdiufauvsJBYystKDfpUwICgqRFHHpaeUA/GnrYZ8r8d7R6TuiY2bEawoKkRQxpTCLBZX5PLUl9YMiOnOs+icSRUEhkkIuP20y6/Y2U9/e63cpnorOHKv+iURRUIikkMtrynEOntmW2q2Ktu5+8jLUokgUBYVICjmtIo/Kwiye2nLE71I8pRZFYikoRFKImXF5TTkrd9TT3Ze6y6O2q48ioRQUIinm8ppyevojvLA9dWdabtN62QmloBBJMefOLCYvM5Syo58GIo6OXq1ul0gKCpEUkxYMcMm8STyz7QgDkdSbTbbj6DxPmr4jYRQUIino8ppyGjv7WL839dbS1qJFiaegEElBF88rIy1oPP76Ib9LGXPHJgRUUCSMgkIkBeVlprGiZjK/fnV/yo1+emuKcV16ShQFhUiK+sSyKlq7+3l0Q2otGtnWffTSk4IiURQUIilqSVURp1Xk85OXdqfUEqnHWhS64S5hFBQiKcrMuHnZDLYdamd1beqsU9HeoxZFoikoRFLYNWdWUpidxk9e2u13KWOmTYsWJZyCQiSFZaYFuX7JdJ7ccoi6lm6/yzllfeEIr+1rISstSFpQP74SRWdaJMXduDS6hvZ/rd7jcyWn5lBrDzf8aDVPbzvCX15Y7Xc5E4qCQiTFTS3K5vKacn7+yl56+sfnUNmXdjZw9fdeYOvBNu7+i7P4/OVz/S5pQlFQiEwAHz+/ipau/nG5TOrrda3cdO8rFGSl8Zs7lnP1wil+lzThKChEJoCl1SWU5mbwh00H/S5lxL715BvkZYZ4+NPLmVOe53c5E5KCQmQCCAaMKxdM5pltR+jsDftdzrC9uqeZZ9+o57YLqynI1nBYvygoRCaIqxZW0NMf4Zlt42f1u3976k1KctL5xPlVfpcyoSkoRCaIJVXFlOWNn8tPL9c2snJHA5++eBY5Gbpnwk8KCpEJIhgw3jtOLj855/jWU28yKS+DG5fO8LucCU9BITKBvPeMCnrDEZ5O8stPL+5o5JVdTdxxyWwy04J+lzPheRYUZnafmR0xs9eH2F5kZo+Y2UYze8XMFnhVi4hELa4qZlJeBn/YmLyXnyIRx788+QYVBZl8dMk0v8sRvG1R3A+8J872vwc2OOcWAh8HvuNhLSJC7PLTGRU8+8YROpL08tMj6+t4bV8Ln798rloTScKzoHDOPQ/Em7KyBngmtu82oMrMyr2qR0SirloYu/yUhDffdfSG+cYft7FoWiHXnT3V73Ikxs8+iteADwKY2bnADGDQ7wwzu83M1prZ2vr6+gSWKJJ6zpleRHl+Br9PwstPdz+zg/r2Xr76vhoCAfO7HInxMyi+ARSa2Qbgs8B6YNCJaJxz9zjnFjvnFpeVlSWyRpGUEwgYK2om88L2BnrDyTP30+6GTu5buYsPnl3JWdOL/C5HjuNbUDjn2pxztzjnziTaR1EG1PpVj8hEcvG8Mrr7B1i7u9nvUo75h99vIS1o/O175vtdipzAt6Aws0IzS499+Sngeedcm1/1iEwkS6tLSA8GeO7N5LiU+8L2ev609QifefccJuVn+l2OnMDL4bE/B1YB88xsv5l90sxuN7PbY7ucBrxuZm8AVwKf86oWEXm7nIwQS2YW8dwb/geFc45vP/UmlYVZ3HpBld/lyCA8uy/eOXfDSbavAjSpvIhPLppbxj/+YRsHW7upKMjyrY5VOxtZv7eFr127gIyQhsMmI92ZLTJBXTR3EgDP+3z56XvP7GBSXgYfPkfDYZOVgkJkgppbnsvk/Exf+ynW7m5iVW0jt11YrZvrkpiCQmSCMjMunFvKyu0NhAcivtTwvWd2UJyTzl+cN92X48vwKChEJrCL5k6irSfMa/tbEn7sjftbeO7Nej71rplkp2sa8WSmoBCZwC6YXUrA8GX0093P7CA/M8RNmkY86SkoRCawguw0zppelPB+ioOt3Ty55TCfWFZFXqaWOE12CgqRCe6iuWVsrGulsaM3Ycc8uhzr+xdNSdgxZfQUFCIT3EVzy3AOXtjekLBjPrvtCFOLspg9KTdhx5TRU1CITHBnVBYwKS+Dx19PzGyyPf0DrNzRwKXzJ2GmGWLHAwWFyAQXOLaYUT1tPf2eH29VbSM9/REumT/J82PJ2FBQiAjvWzSFvnCEpzZ7v5jRM1uPkJUWZGl1iefHkrGhoBARzp5eSGVhFr/beMDT4zjneGbbEZbPLtWd2OOIgkJEMDOuXljBC9sbaO7s8+w42490UNfSzaWn6bLTeKKgEBEgevkpHHH8cfMhz47x9NbosNhL5ikoxhMFhYgAcPqUfGaW5nh6+enZbUc4fUo+kwu0ONF4oqAQEeCty0+rdjZypL1nzD+/pauPtXuaeLdGO407CgoROeZ9i6YQcfD4prG//PTcm/VEHAqKcUhBISLHzC3PY155nieXn/609QglOeksmlo45p8t3lJQiMjbvG9RBWt2N7O3sWvMPrOzN8yfthzmigWTCQR0N/Z4o6AQkbe57pypBAPGz17ZO2af+dSWw3T3D3DtmZVj9pmSOAoKEXmbioIsLp0/iV+u3UdveGBMPvPRDXVUFmaxeEbRmHyeJJaCQkTe4calM2jq7OOPr596p3ZDRy8vbG/g/WdO0WWncUpBISLvcMHsUmaUZPNfq/ec8mf9fuNBBiKOa87U2hPjlYJCRN4hEDA+dt501uxuZtuhtlP6rEc31DF/ch7zJ+ePUXWSaAoKERnUh8+ZRnoowAOrR9+pvaexk/V7W7hGndjjmoJCRAZVlJPO1WdU8Mj6Ojp7w6P6jN9siN6P8X5ddhrXFBQiMqevdDwAAAjqSURBVKSPLZ1BR2+YRzfUjfi9zjke3VDHuTOLqSzM8qA6SRQFhYgM6ezphdRU5HPvC7sID0RG9N7NB9qore/UvRMpwLOgMLP7zOyImb0+xPYCM/utmb1mZpvN7BavahGR0TEz7rx0DrUNnTyyfmStiuferAfgitPLvShNEsjLFsX9wHvibL8D2OKcWwRcDHzLzNI9rEdERuGK08tZUJnPd57eTl94+K2KVTsbmT85j5LcDA+rk0TwLCicc88DTfF2AfLMzIDc2L6j6zETEc+YGV9YMY/9zd38Yu2+Yb2nNzzAmt1NnD9L62KnAj/7KO4GTgMOAJuAzznnBv11xcxuM7O1Zra2vr4+kTWKCHDx3DIWzyji7me209N/8mk9NuxtoTcc4fxqBUUq8DMorgA2AFOAM4G7zWzQO3Kcc/c45xY75xaXlZUlskYR4a1WxeG23mHdrb2qtpGAwXkKipTgZ1DcAjzsonYAu4D5PtYjInGcP6uE5bNL+MGfd570voqXdjZy+pQCCrLSElSdeMnPoNgLXApgZuXAPKDWx3pE5CS+uGIejZ19fPmRTTjnBt2nu2+ADXtbWKb+iZTh5fDYnwOrgHlmtt/MPmlmt5vZ7bFdvgYsM7NNwNPAl5xzDV7VIyKn7qzpRXxxxVwe3XCAHz43+O91r+5ppm8gwlIFRcoIefXBzrkbTrL9ALDCq+OLiDfuuGQ2bxzu4JtPbGPOpFwuq3n7fRKrahsIBowlVcU+VShjTXdmi8iImBnfvG4hC6YU8LkH1/PGofa3bX9pZyOLphaQm+HZ76GSYAoKERmxrPQgP/r4YnIyQtx6/xp2N3QC0NEbZuP+Vt0/kWIUFCIyKpMLMrnv5iV09YX50A9XsflAK2t2NTEQcSybVep3eTKGFBQiMmoLKgv41e3LSAsa1//Hau57cRfpwQDnaG3slKKgEJFTMntSLr/+9DLK8jN4YXsDZ00vJDMt6HdZMoYUFCJyyioLs/jVX53PFaeX8/Hzq/wuR8aYhiWIyJgoyc3gP25a7HcZ4gG1KEREJC4FhYiIxKWgEBGRuBQUIiISl4JCRETiUlCIiEhcCgoREYlLQSEiInHZUKtUJSszqwdOvmjv4AqA1jHcN94+Q20b7uuD7VcKJGpxp5Gcq1N9fyLP9XBeS9XzPJz9R7td53rk+yb6ezrHOVd2kpoG55ybMA/gnrHcN94+Q20b7uuD7QesTcZzNZ7O9XBeS9XzPJz9R7td53rk+46n7+mJdunpt2O8b7x9hto23NdHUqsXTvX4yXquh/taoiTyPA9n/9Fu17ke+b7j5nt63F16msjMbK1zTpPpeEznOXF0rhPjVM/zRGtRjHf3+F3ABKHznDg614lxSudZLQoREYlLLQoREYlLQSEiInEpKEREJC4FRQows2vN7Edm9gszW+F3PanMzKrN7F4z+7XftaQaM8sxs5/Evpc/5nc9qWyk38cKCp+Z2X1mdsTMXj/h9feY2RtmtsPM/jbeZzjnHnXO/SVwO/BRL+sdz8boXNc65z7pbaWpY4Tn/IPAr2Pfy+9PeLHj3EjO9Ui/jxUU/rsfeM/xL5hZEPg+cCVQA9xgZjVmdoaZ/e6Ex6Tj3vo/Yu+Twd3P2J1rGZ77GeY5B6YC+2K7DSSwxlRxP8M/1yMSGovqZPScc8+bWdUJL58L7HDO1QKY2YPANc65fwKuPvEzzMyAbwCPO+fWeVvx+DUW51pGZiTnHNhPNCw2oF9iR2yE53rLSD5b/xjJqZK3frOC6H+gyjj7fxa4DPiQmd3uZWEpaETn2sxKzOyHwFlm9ndeF5eihjrnDwPXmdkP8H8Km1Qx6Lke6fexWhQpwDn3XeC7ftcxETjnGon2BckYc851Arf4XcdEMNLvY7UoklMdMO24r6fGXpOxp3OdeDrniTMm51pBkZzWAHPMbKaZpQPXA4/5XFOq0rlOPJ3zxBmTc62g8JmZ/RxYBcwzs/1m9knnXBj4DPAEsBX4pXNus591pgKd68TTOU8cL8+1JgUUEZG41KIQEZG4FBQiIhKXgkJEROJSUIiISFwKChERiUtBISIicSkoJGWYWUeCj/efo5mJ8xSPeZeZZSfymCK6j0JShpl1OOdyx/DzQrEblhImNhOwOeciQ2zfDSx2zjUksi6Z2NSikJRmZmVm9pCZrYk9lsdeP9fMVpnZejN7yczmxV6/2cweM7NngKfN7GIz+7OZ/drMtpnZA7Ef5sReXxx73mFmXzez18xstZmVx16fFft6k5n9w2CtHjOrii0s81PgdWCamf3AzNaa2WYz+9+x/e4EpgDPmtmzsddWxP4e68zsV2Y2ZkEpcoxzTg89UuIBdAzy2s+AC2LPpwNbY8/zgVDs+WXAQ7HnNxOdirk49vXFQCvRydQCRKdIOPp5fyb62z2AA94Xe/5N4H/Env8OuCH2/PYhaqwCIsDS4147evxg7DgLY1/vBkpjz0uB54Gc2NdfAr7i97+DHqn30DTjkuouA2pijQCA/Nhv3QXAT8xsDtEf8mnHvecp51zTcV+/4pzbD2BmG4j+YF95wnH6iIYCwKvA5bHn5wPXxp7/DPjXIerc45xbfdzXHzGz24guBVBBdHWyjSe8Z2ns9Rdjf790okEmMqYUFJLqAkR/U+85/kUzuxt41jn3gdiqYH8+bnPnCZ/Re9zzAQb/f9PvnHMn2SeeY8c0s5nAF4ElzrlmM7sfyBzkPUY01G4Y4bFERkR9FJLqniS6AiAAZnZm7GkBb83Lf7OHx18NXBd7fv0w35NPNDhaY30dVx63rR3IO+6zl5vZbAAzyzGzuadessjbKSgklWTHplc++vg8cCew2Mw2mtkW3lrV65vAP5nZerxtWd8FfN7MNgKzifZ3xOWcew1YD2wjernqxeM23wP80cyedc7VEw25n8c+fxUwf2zLF9HwWBFPxe556HbOOTO7nmjH9jV+1yUyEuqjEPHWOcDdsSG1LcCtPtcjMmJqUYiISFzqoxARkbgUFCIiEpeCQkRE4lJQiIhIXAoKERGJS0EhIiJx/X/Si23UC8+LygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reset_seed(SEED)\n",
    "trainloader = DataLoader(trainset, batch_size=DESIRED_BATCH_SIZE, shuffle=True)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "model = ConvNet()\n",
    "model = model.to(device)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.5)\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "model, optimizer = amp.initialize(model, optimizer, opt_level='O2')\n",
    "\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device='cuda')\n",
    "lr_finder.range_test(trainloader, end_lr=10, num_iter=100, step_mode='exp')\n",
    "lr_finder.plot()\n",
    "lr_finder.reset()\n",
    "\n",
    "del optimizer, model, lr_finder"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch-lr-finder3",
   "language": "python",
   "name": "pytorch-lr-finder3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
